{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Learn Keras for Deep Neural Networks\n",
    "## Chapter 6 - The Path Foward"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### MNIST example for Convolutional Neural Networks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/jmoolay/anaconda3/lib/python3.6/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n",
      "/Users/jmoolay/anaconda3/lib/python3.6/importlib/_bootstrap.py:219: RuntimeWarning: compiletime version 3.5 of module 'tensorflow.python.framework.fast_tensor_util' does not match runtime version 3.6\n",
      "  return f(*args, **kwds)\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from keras.datasets import mnist\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, Dropout, Flatten\n",
    "from keras.layers.convolutional import Conv2D, MaxPooling2D\n",
    "from keras.utils import np_utils"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Loading data from Keras datasets\n",
    "(x_train, y_train), (x_test, y_test) = mnist.load_data()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Samples: 60000\n",
      "Testing Samples: 10000\n",
      "Height: 28 x Width:28\n"
     ]
    }
   ],
   "source": [
    "#Defining the height and weight and number of samples\n",
    "#Each Image is a 28 x 28 with 1 channel matrix\n",
    "training_samples, height, width = x_train.shape\n",
    "testing_samples,_,_ = x_test.shape\n",
    "\n",
    "print(\"Training Samples:\",training_samples)\n",
    "print(\"Testing Samples:\",testing_samples)\n",
    "print(\"Height: \"+str(height)+\" x Width:\"+ str(width))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Lets have a look at a sample image in the training data\n",
    "plt.imshow(x_train[0],cmap='gray', interpolation='none')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#We now have to engineer the image data into the right form\n",
    "#For CNN, we would need the data in Height x Width X Channels form\n",
    "#Since the image is in grayscale, we will use channel = 1\n",
    "channel =1\n",
    "x_train = x_train.reshape(training_samples, height, width,channel).astype('float32')\n",
    "x_test = x_test.reshape(testing_samples, height, width, channel).astype('float32')\n",
    "\n",
    "#To improve the training process, we would need to standardizee or normalize the values\n",
    "#We can achieve this using a simple divide by 256 for all values\n",
    "x_train = x_train/255\n",
    "x_test =x_test/255\n",
    "\n",
    "#Total number of digits  =10\n",
    "target_classes = 10\n",
    "\n",
    "# numbers 0-9, so ten classes\n",
    "n_classes = 10\n",
    "\n",
    "# convert integer labels into one-hot vectors\n",
    "y_train = np_utils.to_categorical(y_train, n_classes)\n",
    "y_test = np_utils.to_categorical(y_test, n_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Desing the CNN Model\n",
    "model = Sequential()\n",
    "model.add(Conv2D(64, (5, 5), input_shape=(height,width ,1), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Conv2D(64, (3, 3), activation='relu'))\n",
    "model.add(MaxPooling2D(pool_size=(2, 2)))\n",
    "model.add(Dropout(0.25))\n",
    "model.add(Flatten())\n",
    "model.add(Dense(128, activation='relu'))\n",
    "model.add(Dense(n_classes, activation='softmax'))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Compile model\n",
    "model.compile(loss='categorical_crossentropy', optimizer='adam', metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 60000 samples, validate on 10000 samples\n",
      "Epoch 1/10\n",
      "60000/60000 [==============================] - 61s 1ms/step - loss: 0.2452 - acc: 0.9266 - val_loss: 0.0627 - val_acc: 0.9806\n",
      "Epoch 2/10\n",
      "60000/60000 [==============================] - 64s 1ms/step - loss: 0.0651 - acc: 0.9804 - val_loss: 0.0414 - val_acc: 0.9860\n",
      "Epoch 3/10\n",
      "60000/60000 [==============================] - 62s 1ms/step - loss: 0.0457 - acc: 0.9858 - val_loss: 0.0274 - val_acc: 0.9912\n",
      "Epoch 4/10\n",
      "60000/60000 [==============================] - 62s 1ms/step - loss: 0.0374 - acc: 0.9885 - val_loss: 0.0288 - val_acc: 0.9907\n",
      "Epoch 5/10\n",
      "60000/60000 [==============================] - 61s 1ms/step - loss: 0.0298 - acc: 0.9908 - val_loss: 0.0280 - val_acc: 0.9904\n",
      "Epoch 6/10\n",
      "60000/60000 [==============================] - 62s 1ms/step - loss: 0.0259 - acc: 0.9917 - val_loss: 0.0254 - val_acc: 0.9918\n",
      "Epoch 7/10\n",
      "60000/60000 [==============================] - 64s 1ms/step - loss: 0.0218 - acc: 0.9930 - val_loss: 0.0224 - val_acc: 0.9924\n",
      "Epoch 8/10\n",
      "60000/60000 [==============================] - 60s 999us/step - loss: 0.0212 - acc: 0.9931 - val_loss: 0.0233 - val_acc: 0.9923\n",
      "Epoch 9/10\n",
      "60000/60000 [==============================] - 58s 963us/step - loss: 0.0172 - acc: 0.9943 - val_loss: 0.0284 - val_acc: 0.9904\n",
      "Epoch 10/10\n",
      "60000/60000 [==============================] - 56s 930us/step - loss: 0.0149 - acc: 0.9949 - val_loss: 0.0204 - val_acc: 0.9936\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb18d3bf28>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Fit the model\n",
    "model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=10, batch_size=200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loss = 0.02039033946258933\n",
      "acc = 0.9936\n"
     ]
    }
   ],
   "source": [
    "#Finally let's evaluate the model performance\n",
    "metrics = model.evaluate(x_test, y_test, verbose=0)\n",
    "for i in range(0,len(model.metrics_names)):\n",
    "    print(str(model.metrics_names[i])+\" = \"+str(metrics[i]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### IMDB text classification example for RNN/LSTM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import the necessary packages\n",
    "from keras.datasets import imdb\n",
    "from keras.models import Sequential\n",
    "from keras.layers import Dense, LSTM\n",
    "from keras.layers.embeddings import Embedding\n",
    "from keras.preprocessing import sequence\n",
    "\n",
    "#Setting a max cap for the number of distinct words\n",
    "top_words = 5000\n",
    "#Loading the training and test data from keras datasets\n",
    "(x_train, y_train), (x_test, y_test) = imdb.load_data(num_words=top_words)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Since the length of each text will be varying\n",
    "#We will pad the sequences (i.e. text) to get a uniform length throughout \n",
    "max_text_length = 500\n",
    "x_train = sequence.pad_sequences(x_train, maxlen=max_text_length)\n",
    "x_test = sequence.pad_sequences(x_test, maxlen=max_text_length)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 25000 samples, validate on 25000 samples\n",
      "Epoch 1/3\n",
      "25000/25000 [==============================] - 226s 9ms/step - loss: 0.5312 - acc: 0.7176 - val_loss: 0.5133 - val_acc: 0.7521\n",
      "Epoch 2/3\n",
      "25000/25000 [==============================] - 219s 9ms/step - loss: 0.3137 - acc: 0.8710 - val_loss: 0.3116 - val_acc: 0.8737\n",
      "Epoch 3/3\n",
      "25000/25000 [==============================] - 223s 9ms/step - loss: 0.2616 - acc: 0.8986 - val_loss: 0.3872 - val_acc: 0.8277\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0xb240ccf60>"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Design the network\n",
    "embedding_length = 32\n",
    "model = Sequential()\n",
    "model.add(Embedding(top_words, embedding_length, input_length=max_text_length))\n",
    "model.add(LSTM(100))\n",
    "model.add(Dense(1, activation='sigmoid'))\n",
    "\n",
    "#Compile the  model\n",
    "model.compile(loss='binary_crossentropy', optimizer='adam', metrics=['accuracy'])\n",
    "\n",
    "#Fit the model\n",
    "model.fit(x_train, y_train, validation_data=(x_test, y_test), epochs=3, batch_size=64)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy: 0.73584\n"
     ]
    }
   ],
   "source": [
    "#Evaluate the accuracy on the test dataset\n",
    "scores = model.evaluate(x_test, y_test, verbose=0)\n",
    "print(\"Accuracy:\",scores[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Further readling links\n",
    "Consolidating few reading links and additional references for the readers of the book to explore additional references and other advanced topics "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recommeded books for Machine Learning with Python\n",
    "\n",
    "* Mastering Machine Learning with Python in 6 steps - https://www.apress.com/us/book/9781484228654\n",
    "* Pactical Machine Learning with Python - https://www.apress.com/us/book/9781484232064\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Recommeded books for Deep Learning with Python/Tensorflow\n",
    "* Applied Deep Learning -  https://www.apress.com/us/book/9781484237892\n",
    "* Pro Deep Learning with Tensorflow - https://www.apress.com/us/book/9781484230954\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Links Convolutional Neural Networks (CNN)\n",
    "* https://adeshpande3.github.io/adeshpande3.github.io/A-Beginner's-Guide-To-Understanding-Convolutional-Neural-Networks/\n",
    "* https://medium.freecodecamp.org/an-intuitive-guide-to-convolutional-neural-networks-260c2de0a050\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Links for Recurrent Neural Networks\n",
    "* https://colah.github.io/posts/2015-08-Understanding-LSTMs/\n",
    "* https://medium.com/mlreview/understanding-lstm-and-its-diagrams-37e2f46f1714\n",
    "* https://towardsdatascience.com/illustrated-guide-to-lstms-and-gru-s-a-step-by-step-explanation-44e9eb85bf21\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Links for Image Captioning  (CNN + RNN)\n",
    "* https://towardsdatascience.com/image-captioning-in-deep-learning-9cd23fb4d8d2\n",
    "* https://machinelearningmastery.com/introduction-neural-machine-translation/\n",
    "* https://towardsdatascience.com/neural-machine-translation-with-python-c2f0a34f7dd\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Reading Links for Generative Adversarial Networks \n",
    "* https://medium.com/@awjuliani/generative-adversarial-networks-explained-with-a-classic-spongebob-squarepants-episode-54deab2fce39\n",
    "* https://medium.com/ai-society/gans-from-scratch-1-a-deep-introduction-with-code-in-pytorch-and-tensorflow-cb03cdcdba0f\n",
    "* https://towardsdatascience.com/understanding-generative-adversarial-networks-4dafc963f2ef"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
